#!/bin/bash
##################################################################
#总的恶意IP更新升级入口
#此脚本主要是对恶代的的恶意IP地址库进行升级
#当前策略（20141211）是
#从neu、openbl、DRG、nothink中分别去获取相关的ip，然后
#对恶代库已有的进行升级，升级后生成统计日志ChangeLog
#所用到的地址总览
##################################################################
#NEU（东北大学：http://antivirus.neu.edu.cn）
##################################################################
#port scan（http://antivirus.neu.edu.cn/scan/scanlist.php）
#ssh（http://antivirus.neu.edu.cn/ssh/lists/neu.txt）
#smtp（http://antivirus.neu.edu.cn/ssh/lists/neu_smtp.txt）
##################################################################
#OPENBL（http://www.openbl.org）
##################################################################
#FTP attacks（http://www.openbl.org/lists/base_all_ftp-only.txt）
#HTTP scans（http://www.openbl.org/lists/base_all_http-only.txt）
#MAIL attacks（http://www.openbl.org/lists/base_all_mail-only.txt）
#SMTP attacks（http://www.openbl.org/lists/base_all_smtp-only.txt）
#SSH attacks（http://www.openbl.org/lists/base_all_ssh-only.txt）
##################################################################
#DRG（http://www.dragonresearchgroup.org/insight/）
##################################################################
#VNC probe（http://www.dragonresearchgroup.org/insight/vncprobe.txt）
#SSH attacks（http://www.dragonresearchgroup.org/insight/sshpwauth.txt）
##################################################################
#nothink（http://www.nothink.org）
##################################################################
#SSH attacks（http://www.nothink.org/blacklist/blacklist_ssh_all.txt）
##################################################################

#针对NEU进行更新
web_site="http://antivirus.neu.edu.cn/scan/scanlist.php"
neu_port_scan_file=`basename $web_site`
neu_formated_file="neu_"$neu_port_scan_file".txt"
inx=0
file_list[$inx]=$neu_formated_file
inx=$inx+1
wget $web_site && cat $neu_port_scan_file | awk 'BEGIN{print "##Port Scan;S_IP";cmdline="date \"+#NEU(%b %d %Y)\"";system(cmdline);} $1~/^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' > $neu_formated_file;rm -rf $neu_port_scan_file
web_site="http://antivirus.neu.edu.cn/ssh/lists/neu.txt"
neu_ssh_attack_file=`basename $web_site`
neu_formated_file="neu_"$neu_ssh_attack_file".txt"
file_list[$inx]=$neu_formated_file
inx=$inx+1
wget $web_site && cat $neu_ssh_attack_file | awk 'BEGIN{print "##SHH Brute Force Attack;S_IP";cmdline="date \"+#NEU(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $neu_formated_file;rm -rf $neu_ssh_attack_file
web_site="http://antivirus.neu.edu.cn/ssh/lists/neu_smtp.txt"
neu_smtp_attack_file=`basename $web_site`
neu_formated_file="neu_"$neu_smtp_attack_file".txt"
file_list[$inx]=$neu_formated_file
inx=$inx+1
wget $web_site && cat $neu_smtp_attack_file  | awk 'BEGIN{print "##SHH Brute Force Attack;S_IP";cmdline="date \"+#NEU(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $neu_formated_file;rm -rf $neu_smtp_attack_file
#针对OPENBL进行更新
web_site="http://www.openbl.org/lists/base_all_ftp-only.txt"
openbl_ftp_attack_file=`basename $web_site`
openbl_formated_file="openbl_"$openbl_ftp_attack_file".txt"
file_list[$inx]=$openbl_formated_file
inx=$inx+1
wget $web_site && cat $openbl_ftp_attack_file | awk 'BEGIN{print "##SMTP Attack;S_IP";cmdline="date \"+#OPENBL(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $openbl_formated_file;rm -rf $openbl_ftp_attack_file
web_site="http://www.openbl.org/lists/base_all_http-only.txt"
openbl_http_scan_file=`basename $web_site`
openbl_formated_file="openbl_"$openbl_http_scan_file".txt"
file_list[$inx]=$openbl_formated_file
inx=$inx+1
wget $web_site  && cat $openbl_http_scan_file | awk 'BEGIN{print "##HTTP Attack;S_IP";cmdline="date \"+#OPENBL(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $openbl_formated_file;rm -rf $openbl_http_scan_file
web_site="http://www.openbl.org/lists/base_all_mail-only.txt"
openbl_mail_attack_file=`basename $web_site`
openbl_formated_file="openbl_"$openbl_mail_attack_file".txt"
file_list[$inx]=$openbl_formated_file
inx=$inx+1
wget $web_site && cat $openbl_mail_attack_file | awk 'BEGIN{print "##MAIL Attack;S_IP";cmdline="date \"+#OPENBL(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $openbl_formated_file;rm -rf $openbl_mail_attack_file
web_site="http://www.openbl.org/lists/base_all_smtp-only.txt"
openbl_smtp_attack_file=`basename $web_site`
openbl_formated_file="openbl_"$openbl_smtp_attack_file".txt"
file_list[$inx]=$openbl_formated_file
inx=$inx+1
wget $web_site && cat $openbl_smtp_attack_file | awk 'BEGIN{print "##SMTP Attack;S_IP";cmdline="date \"+#OPENBL(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $openbl_formated_file;rm -rf $openbl_smtp_attack_file
web_site="http://www.openbl.org/lists/base_all_ssh-only.txt"
openbl_ssh_attack_file=`basename $web_site`
openbl_formated_file="openbl_"$openbl_ssh_attack_file".txt"
file_list[$inx]=$openbl_formated_file
inx=$inx+1
wget $web_site && cat $openbl_ssh_attack_file | awk 'BEGIN{print "##SHH Brute Force Attack;S_IP";cmdline="date \"+#OPENBL(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $openbl_formated_file;rm -rf $openbl_ssh_attack_file
#针对DRG进行更新
web_site="http://www.dragonresearchgroup.org/insight/vncprobe.txt"
drg_vnc_probe_file=`basename $web_site`
drg_formated_file="drg_"$drg_vnc_probe_file".txt"
file_list[$inx]=$drg_formated_file
inx=$inx+1
wget $web_site && cat $drg_vnc_probe_file | awk -F '|' '{print $3;}' - | sed 's/ //g' - | awk 'BEGIN{print "##VNC Probe;S_IP";cmdline="date \"+#DRG(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $drg_formated_file;rm -rf $drg_vnc_probe_file
web_site="http://www.dragonresearchgroup.org/insight/sshpwauth.txt"
drg_ssh_attack_file=`basename $web_site`
drg_formated_file="drg_"$drg_ssh_attack_file".txt"
file_list[$inx]=$drg_formated_file
inx=$inx+1
wget $web_site && cat $drg_ssh_attack_file | awk -F '|' '{print $3;}' - | sed 's/ //g' - | awk 'BEGIN{print "##SHH Brute Force Attack;S_IP";cmdline="date \"+#DRG(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $drg_formated_file;rm -rf $drg_ssh_attack_file
#针对nothink进行更新
web_site="http://www.nothink.org/blacklist/blacklist_ssh_all.txt"
nothink_ssh_attack_file=`basename $web_site`
nothink_formated_file="nothink_"$nothink_ssh_attack_file".txt"
file_list[$inx]=$drg_formated_file
inx=$inx+1
wget $web_site && cat $nothink_ssh_attack_file | awk 'BEGIN{print "#SHH Brute Force Attack;S_IP";cmdline="date \"+#NoThink(%b %d %Y)\"";system(cmdline);} $1~/[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+/{print $1}' - > $nothink_formated_file;rm -rf $nothink_ssh_attack_file
#接下来就是传入到perl脚本中进行进一步的聚合操作，同时将日志整理出来
#首先来获取下当前路径
cd `dirname $0`
self_path=`pwd`
#这个传入参数必须保证有myself_malip_data.txt作为自分析基础模板，所有的新增ip也是针对这个模板的更新。打印出来的报告只是相对该模板所做的一些改变。
#IP进行分析，然后不能有malip_new.txt
perl malip_update_process.plx ${file_list[@]:0}
